package com.shujia.spark.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo9Union {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("reduceByKey")

    val sc = new SparkContext(conf)

    val rdd1: RDD[Int] = sc.parallelize(List(1, 2, 3, 4, 5))
    val rdd2: RDD[Int] = sc.parallelize(List(4, 5, 6, 7, 8))

    /**
     * union : 将两个相同类型的rd合并成一个rdd
     * union是逻辑层面合并，再底层数据层面没有合并
     */
    val unionRDD: RDD[Int] = rdd1.union(rdd2)

    unionRDD.foreach(println)


    /**
     * distinct:对数据去重
     *
     */

    val distinctRDD: RDD[Int] = unionRDD.distinct()

    distinctRDD.foreach(println)
  }

}
